{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip3 install xlnet-tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlnet\n",
    "import xlnet.modeling, xlnet.xlnet\n",
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "xlnet.xlnet.XLNetModel"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xlnet.xlnet.XLNetModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "SOURCE_DIR = os.path.dirname(os.path.abspath(__name__))\n",
    "sys.path.insert(0, f\"{SOURCE_DIR}/src\")\n",
    "sys.path.append(SOURCE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import itertools\n",
    "import os.path\n",
    "import time\n",
    "import shutil\n",
    "import re\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import torch.optim.lr_scheduler\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import evaluate\n",
    "import trees_newline as trees\n",
    "import vocabulary\n",
    "import nkutil\n",
    "import parse_nk_xlnet_base as parse_nk\n",
    "tokens = parse_nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if parse_nk.use_cuda:\n",
    "    info = torch.load('models/xlnet_dev=83.30.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = parse_nk.NKChartParser.from_spec(info['spec'], info['state_dict'])\n",
    "bert_model = info['spec']['hparams']['bert_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "sd = parser.state_dict()\n",
    "\n",
    "LABEL_VOCAB = [x[0] for x in sorted(parser.label_vocab.indices.items(), key=lambda x: x[1])]\n",
    "TAG_VOCAB = [x[0] for x in sorted(parser.tag_vocab.indices.items(), key=lambda x: x[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "position_table = tf.constant(sd['embedding.position_table'].cpu().numpy(), name=\"position_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_layer_norm(input, torch_name, name):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=tf.constant(sd[f'{torch_name}.b_2'].cpu().numpy(), name=f\"{name}/offset\"),\n",
    "        scale=tf.constant(sd[f'{torch_name}.a_2'].cpu().numpy(), name=f\"{name}/scale\"),\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "\n",
    "def make_heads(input, shape_bthf, shape_xtf, torch_name, name):\n",
    "    res = tf.matmul(input,\n",
    "        tf.constant(sd[torch_name].cpu().numpy().transpose((1,0,2)).reshape((512, -1)), name=f\"{name}/W\"))\n",
    "    res = tf.reshape(res, shape_bthf)\n",
    "    res = tf.transpose(res, (0,2,1,3)) # batch x num_heads x time x feat\n",
    "    res = tf.reshape(res, shape_xtf) # _ x time x feat\n",
    "    return res\n",
    "\n",
    "def make_attention(input, nonpad_ids, dim_flat, dim_padded, valid_mask, torch_name, name):\n",
    "    input_flat = tf.scatter_nd(indices=nonpad_ids[:, None], updates=input, shape=tf.concat([dim_flat, tf.shape(input)[1:]], axis=0))\n",
    "    input_flat_dat, input_flat_pos = tf.split(input_flat, 2, axis=-1)\n",
    "\n",
    "    shape_bthf = tf.concat([dim_padded, [8, -1]], axis=0)\n",
    "    shape_bhtf = tf.convert_to_tensor([dim_padded[0], 8, dim_padded[1], -1])\n",
    "    shape_xtf = tf.convert_to_tensor([dim_padded[0] * 8, dim_padded[1], -1])\n",
    "    shape_xf = tf.concat([dim_flat, [-1]], axis=0)\n",
    "\n",
    "    qs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_qs1', f'{name}/q_dat')\n",
    "    ks1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_ks1', f'{name}/k_dat')\n",
    "    vs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_vs1', f'{name}/v_dat')\n",
    "    qs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_qs2', f'{name}/q_pos')\n",
    "    ks2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_ks2', f'{name}/k_pos')\n",
    "    vs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_vs2', f'{name}/v_pos')\n",
    "\n",
    "    qs = tf.concat([qs1, qs2], axis=-1)\n",
    "    ks = tf.concat([ks1, ks2], axis=-1)\n",
    "    attn_logits = tf.matmul(qs, ks, transpose_b=True) / (1024 ** 0.5)\n",
    "\n",
    "    attn_mask = tf.reshape(tf.tile(valid_mask, [1,8*dim_padded[1]]), tf.shape(attn_logits))\n",
    "    # TODO(nikita): use tf.where and -float('inf') here?\n",
    "    attn_logits -= 1e10 * tf.to_float(~attn_mask)\n",
    "\n",
    "    attn = tf.nn.softmax(attn_logits)\n",
    "\n",
    "    attended_dat_raw = tf.matmul(attn, vs1)\n",
    "    attended_dat_flat = tf.reshape(tf.transpose(tf.reshape(attended_dat_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_dat = tf.gather(attended_dat_flat, nonpad_ids)\n",
    "    attended_pos_raw = tf.matmul(attn, vs2)\n",
    "    attended_pos_flat = tf.reshape(tf.transpose(tf.reshape(attended_pos_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_pos = tf.gather(attended_pos_flat, nonpad_ids)\n",
    "\n",
    "    out_dat = tf.matmul(attended_dat, tf.constant(sd[f'{torch_name}.proj1.weight'].cpu().numpy().transpose()))\n",
    "    out_pos = tf.matmul(attended_pos, tf.constant(sd[f'{torch_name}.proj2.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_dense_relu_dense(input, torch_name, torch_type, name):\n",
    "    # TODO: use name\n",
    "    mul1 = tf.matmul(input, tf.constant(sd[f'{torch_name}.w_1{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul1b = tf.nn.bias_add(mul1, tf.constant(sd[f'{torch_name}.w_1{torch_type}.bias'].cpu().numpy()))\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, tf.constant(sd[f'{torch_name}.w_2{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul2b = tf.nn.bias_add(mul2, tf.constant(sd[f'{torch_name}.w_2{torch_type}.bias'].cpu().numpy()))\n",
    "    return mul2b\n",
    "\n",
    "def make_ff(input, torch_name, name):\n",
    "    # TODO: use name\n",
    "    input_dat, input_pos = tf.split(input, 2, axis=-1)\n",
    "    out_dat = make_dense_relu_dense(input_dat, torch_name, 'c', name=\"TODO_dat\")\n",
    "    out_pos = make_dense_relu_dense(input_pos, torch_name, 'p', name=\"TODO_pos\")\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_stacks(input, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks):\n",
    "    res = input\n",
    "    for i in range(num_stacks):\n",
    "        res = make_attention(res, nonpad_ids, dim_flat, dim_padded, valid_mask, f'encoder.attn_{i}', name=f'attn_{i}')\n",
    "        res = make_ff(res, f'encoder.ff_{i}', name=f'ff_{i}')\n",
    "    return res\n",
    "\n",
    "def make_layer_norm_with_constants(input, constants):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=constants[0],\n",
    "        scale=constants[1],\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "def make_flabel_with_constants(input, constants):\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='flabel')\n",
    "    return mul2b\n",
    "\n",
    "def make_ftag(input):\n",
    "    constants = (\n",
    "        tf.constant(sd['f_tag.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_tag.1.b_2'].cpu().numpy(), name=\"tag/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_tag.1.a_2'].cpu().numpy(), name=\"tag/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_tag.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='ftag')\n",
    "    return mul2b\n",
    "\n",
    "def make_flabel_constants():\n",
    "    return (\n",
    "        tf.constant(sd['f_label.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_label.1.b_2'].cpu().numpy(), name=\"label/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_label.1.a_2'].cpu().numpy(), name=\"label/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_label.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "\n",
    "def make_network():\n",
    "    # batch x num_subwords\n",
    "    input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32, name='input_ids')\n",
    "    word_end_mask = tf.placeholder(shape=(None, None), dtype=tf.int32, name='word_end_mask')\n",
    "    input_dat, nonpad_ids, dim_flat, dim_padded, valid_mask, lengths, bert_features = make_bert(input_ids, word_end_mask)\n",
    "    input_pos_flat = tf.tile(position_table[:dim_padded[1]], [dim_padded[0], 1])\n",
    "    input_pos = tf.gather(input_pos_flat, nonpad_ids)\n",
    "\n",
    "    input_joint = tf.concat([input_dat, input_pos], -1)\n",
    "    input_joint = make_layer_norm(input_joint, 'embedding.layer_norm', 'embedding/layer_norm')\n",
    "\n",
    "    word_out = make_stacks(input_joint, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks=parser.spec['hparams']['num_layers'])\n",
    "    word_out = tf.concat([word_out[:, 0::2], word_out[:, 1::2]], -1)\n",
    "\n",
    "    # part-of-speech predictions\n",
    "    ftag = make_ftag(word_out)\n",
    "    tags_packed = tf.argmax(ftag, axis=-1)\n",
    "    tags = tf.reshape(\n",
    "        tf.scatter_nd(indices=nonpad_ids[:, None], updates=tags_packed, shape=dim_flat),\n",
    "        dim_padded\n",
    "        )\n",
    "    tags = tf.identity(tags, name=\"tags\")\n",
    "\n",
    "    fp_out = tf.concat([word_out[:-1,:512], word_out[1:,512:]], -1)\n",
    "\n",
    "    fp_start_idxs = tf.cumsum(lengths, exclusive=True)\n",
    "    fp_end_idxs = tf.cumsum(lengths) - 1 # the number of fenceposts is 1 less than the number of words\n",
    "\n",
    "    fp_end_idxs_uneven = fp_end_idxs - tf.convert_to_tensor([1, 0])\n",
    "\n",
    "    # Have to make these outside tf.map_fn for model compression to work\n",
    "    constants = make_flabel_constants()\n",
    "\n",
    "    def to_map(start_and_end):\n",
    "        start, end = start_and_end\n",
    "        fp = fp_out[start:end]\n",
    "        flabel = make_flabel_with_constants(tf.reshape(fp[None,:,:] - fp[:,None,:], (-1, 1024)), constants)\n",
    "        actual_chart_size = end-start\n",
    "        flabel = tf.reshape(flabel, [actual_chart_size, actual_chart_size, -1])\n",
    "        amount_to_pad = dim_padded[1] - actual_chart_size\n",
    "        # extra padding on the label dimension is for the not-a-constituent label,\n",
    "        # which always has a score of 0\n",
    "        flabel = tf.pad(flabel, [[0, amount_to_pad], [0, amount_to_pad], [1, 0]])\n",
    "        return flabel\n",
    "\n",
    "    charts = tf.map_fn(to_map, (fp_start_idxs, fp_end_idxs), dtype=(tf.float32))\n",
    "    charts = tf.identity(charts, name=\"charts\")\n",
    "\n",
    "    return input_ids, word_end_mask, charts, tags, bert_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_tf_xlnet_to_pytorch_map(model, config, tf_weights=None):\n",
    "    \"\"\" A map of modules from TF to PyTorch.\n",
    "        I use a map to keep the PyTorch model as\n",
    "        identical to the original PyTorch model as possible.\n",
    "    \"\"\"\n",
    "\n",
    "    tf_to_pt_map = {}\n",
    "\n",
    "    if hasattr(model, \"transformer\"):\n",
    "        if hasattr(model, \"lm_loss\"):\n",
    "            # We will load also the output bias\n",
    "            tf_to_pt_map[\"model/lm_loss/bias\"] = model.lm_loss.bias\n",
    "        if hasattr(model, \"sequence_summary\") and \"model/sequnece_summary/summary/kernel\" in tf_weights:\n",
    "            # We will load also the sequence summary\n",
    "            tf_to_pt_map[\"model/sequnece_summary/summary/kernel\"] = model.sequence_summary.summary.weight\n",
    "            tf_to_pt_map[\"model/sequnece_summary/summary/bias\"] = model.sequence_summary.summary.bias\n",
    "        if (\n",
    "            hasattr(model, \"logits_proj\")\n",
    "            and config.finetuning_task is not None\n",
    "            and \"model/regression_{}/logit/kernel\".format(config.finetuning_task) in tf_weights\n",
    "        ):\n",
    "            tf_to_pt_map[\"model/regression_{}/logit/kernel\".format(config.finetuning_task)] = model.logits_proj.weight\n",
    "            tf_to_pt_map[\"model/regression_{}/logit/bias\".format(config.finetuning_task)] = model.logits_proj.bias\n",
    "\n",
    "        # Now load the rest of the transformer\n",
    "        model = model.transformer\n",
    "\n",
    "    # Embeddings and output\n",
    "    tf_to_pt_map.update(\n",
    "        {\n",
    "            \"model/transformer/word_embedding/lookup_table\": model.word_embedding.weight,\n",
    "            \"model/transformer/mask_emb/mask_emb\": model.mask_emb,\n",
    "        }\n",
    "    )\n",
    "\n",
    "    # Transformer blocks\n",
    "    for i, b in enumerate(model.layer):\n",
    "        layer_str = \"model/transformer/layer_%d/\" % i\n",
    "        tf_to_pt_map.update(\n",
    "            {\n",
    "                layer_str + \"rel_attn/LayerNorm/gamma\": b.rel_attn.layer_norm.weight,\n",
    "                layer_str + \"rel_attn/LayerNorm/beta\": b.rel_attn.layer_norm.bias,\n",
    "                layer_str + \"rel_attn/o/kernel\": b.rel_attn.o,\n",
    "                layer_str + \"rel_attn/q/kernel\": b.rel_attn.q,\n",
    "                layer_str + \"rel_attn/k/kernel\": b.rel_attn.k,\n",
    "                layer_str + \"rel_attn/r/kernel\": b.rel_attn.r,\n",
    "                layer_str + \"rel_attn/v/kernel\": b.rel_attn.v,\n",
    "                layer_str + \"ff/LayerNorm/gamma\": b.ff.layer_norm.weight,\n",
    "                layer_str + \"ff/LayerNorm/beta\": b.ff.layer_norm.bias,\n",
    "                layer_str + \"ff/layer_1/kernel\": b.ff.layer_1.weight,\n",
    "                layer_str + \"ff/layer_1/bias\": b.ff.layer_1.bias,\n",
    "                layer_str + \"ff/layer_2/kernel\": b.ff.layer_2.weight,\n",
    "                layer_str + \"ff/layer_2/bias\": b.ff.layer_2.bias,\n",
    "            }\n",
    "        )\n",
    "\n",
    "    # Relative positioning biases\n",
    "    if config.untie_r:\n",
    "        r_r_list = []\n",
    "        r_w_list = []\n",
    "        r_s_list = []\n",
    "        seg_embed_list = []\n",
    "        for b in model.layer:\n",
    "            r_r_list.append(b.rel_attn.r_r_bias)\n",
    "            r_w_list.append(b.rel_attn.r_w_bias)\n",
    "            r_s_list.append(b.rel_attn.r_s_bias)\n",
    "            seg_embed_list.append(b.rel_attn.seg_embed)\n",
    "    else:\n",
    "        r_r_list = [model.r_r_bias]\n",
    "        r_w_list = [model.r_w_bias]\n",
    "        r_s_list = [model.r_s_bias]\n",
    "        seg_embed_list = [model.seg_embed]\n",
    "    tf_to_pt_map.update(\n",
    "        {\n",
    "            \"model/transformer/r_r_bias\": r_r_list,\n",
    "            \"model/transformer/r_w_bias\": r_w_list,\n",
    "            \"model/transformer/r_s_bias\": r_s_list,\n",
    "            \"model/transformer/seg_embed\": seg_embed_list,\n",
    "        }\n",
    "    )\n",
    "    return tf_to_pt_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert(input_ids, word_end_mask):\n",
    "    # We can derive input_mask from either input_ids or word_end_mask\n",
    "    input_mask = (1 - tf.cumprod(1 - word_end_mask, axis=-1, reverse=True))\n",
    "    input_mask = (1 - input_mask)\n",
    "    # input_mask = word_end_mask\n",
    "    token_type_ids = tf.zeros_like(input_ids)\n",
    "    bert_model = make_bert_instance(input_ids, input_mask, token_type_ids)\n",
    "\n",
    "    bert_features = bert_model.get_sequence_output()\n",
    "    print(bert_features)\n",
    "    bert_features = tf.transpose(bert_features, perm=[1, 0, 2])\n",
    "    bert_features_packed = tf.gather(\n",
    "        tf.reshape(bert_features, [-1, int(bert_features.shape[-1])]),\n",
    "        tf.to_int32(tf.where(tf.reshape(word_end_mask, (-1,))))[:,0])\n",
    "    projected_annotations = tf.matmul(\n",
    "        bert_features_packed,\n",
    "        tf.constant(sd['project_bert.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    # input_mask is over subwords, whereas valid_mask is over words\n",
    "    sentence_lengths = tf.reduce_sum(word_end_mask, -1)\n",
    "    valid_mask = (tf.range(tf.reduce_max(sentence_lengths))[None,:] < sentence_lengths[:, None])\n",
    "    dim_padded = tf.shape(valid_mask)[:2]\n",
    "    mask_flat = tf.reshape(valid_mask, (-1,))\n",
    "    dim_flat = tf.shape(mask_flat)[:1]\n",
    "    nonpad_ids = tf.to_int32(tf.where(mask_flat)[:,0])\n",
    "\n",
    "    return projected_annotations, nonpad_ids, dim_flat, dim_padded, valid_mask, sentence_lengths, bert_features\n",
    "\n",
    "def make_bert_instance(input_ids, input_mask, token_type_ids):\n",
    "    # Transfer BERT config into tensorflow implementation\n",
    "    import json\n",
    "    \n",
    "    input_ids = tf.transpose(input_ids, [1, 0])\n",
    "    token_type_ids = tf.transpose(token_type_ids, [1, 0])\n",
    "    input_mask = tf.transpose(input_mask, [1, 0])\n",
    "    \n",
    "    c = parser.bert.config.to_dict()\n",
    "    c[\"n_token\"] = 32000\n",
    "    with open('xlnet-config.json', 'w') as fopen:\n",
    "        json.dump(c, fopen)\n",
    "\n",
    "    config = xlnet.xlnet.XLNetConfig(json_path = 'xlnet-config.json')\n",
    "\n",
    "    kwargs = dict(\n",
    "          is_training=False,\n",
    "          use_tpu=False,\n",
    "          use_bfloat16=False,\n",
    "          dropout=0,\n",
    "          dropatt=0,\n",
    "          init='normal',\n",
    "          init_range=0.1,\n",
    "          init_std=0.02,\n",
    "          clamp_len=-1)\n",
    "    \n",
    "    xlnet_parameters = xlnet.xlnet.RunConfig(**kwargs)\n",
    "    model = xlnet.xlnet.XLNetModel(\n",
    "        xlnet_config=config,\n",
    "        run_config=xlnet_parameters,\n",
    "        input_ids=input_ids,\n",
    "        seg_ids=token_type_ids,\n",
    "        input_mask=tf.cast(input_mask, tf.float32))\n",
    "    \n",
    "    mapping = build_tf_xlnet_to_pytorch_map(parser.bert, parser.bert.config)\n",
    "    \n",
    "    bert_variables = [v for v in tf.get_collection('variables')]\n",
    "    tf.variables_initializer(bert_variables).run()\n",
    "    \n",
    "    for i in range(len(bert_variables)):\n",
    "        variable = bert_variables[i]\n",
    "        name = variable.name.split(':')[0]\n",
    "        print(name)\n",
    "        if \"kernel\" in name and (\"ff\" in name or \"summary\" in name or \"logit\" in name):\n",
    "            print(\"Transposing\")\n",
    "            array = mapping[name].T\n",
    "        elif isinstance(mapping[name], list):\n",
    "            print(mapping[name][0].shape)\n",
    "            c = [torch.unsqueeze(t, 0) for t in mapping[name]]\n",
    "            print(c[0].shape)\n",
    "            array = torch.cat(c, 0)\n",
    "        else:\n",
    "            array = mapping[name]\n",
    "            \n",
    "        variable.load(array.detach().cpu().numpy())\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/xlnet.py:64: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/xlnet.py:221: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/xlnet.py:221: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/modeling.py:453: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/modeling.py:460: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/modeling.py:535: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/xlnet/modeling.py:67: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "model/transformer/r_w_bias\n",
      "torch.Size([12, 64])\n",
      "torch.Size([1, 12, 64])\n",
      "WARNING:tensorflow:From <ipython-input-14-7df9c2cf3ecb>:83: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n",
      "model/transformer/r_r_bias\n",
      "torch.Size([12, 64])\n",
      "torch.Size([1, 12, 64])\n",
      "model/transformer/word_embedding/lookup_table\n",
      "model/transformer/r_s_bias\n",
      "torch.Size([12, 64])\n",
      "torch.Size([1, 12, 64])\n",
      "model/transformer/seg_embed\n",
      "torch.Size([2, 12, 64])\n",
      "torch.Size([1, 2, 12, 64])\n",
      "model/transformer/layer_0/rel_attn/q/kernel\n",
      "model/transformer/layer_0/rel_attn/k/kernel\n",
      "model/transformer/layer_0/rel_attn/v/kernel\n",
      "model/transformer/layer_0/rel_attn/r/kernel\n",
      "model/transformer/layer_0/rel_attn/o/kernel\n",
      "model/transformer/layer_0/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_0/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_0/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_0/ff/layer_1/bias\n",
      "model/transformer/layer_0/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_0/ff/layer_2/bias\n",
      "model/transformer/layer_0/ff/LayerNorm/beta\n",
      "model/transformer/layer_0/ff/LayerNorm/gamma\n",
      "model/transformer/layer_1/rel_attn/q/kernel\n",
      "model/transformer/layer_1/rel_attn/k/kernel\n",
      "model/transformer/layer_1/rel_attn/v/kernel\n",
      "model/transformer/layer_1/rel_attn/r/kernel\n",
      "model/transformer/layer_1/rel_attn/o/kernel\n",
      "model/transformer/layer_1/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_1/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_1/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_1/ff/layer_1/bias\n",
      "model/transformer/layer_1/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_1/ff/layer_2/bias\n",
      "model/transformer/layer_1/ff/LayerNorm/beta\n",
      "model/transformer/layer_1/ff/LayerNorm/gamma\n",
      "model/transformer/layer_2/rel_attn/q/kernel\n",
      "model/transformer/layer_2/rel_attn/k/kernel\n",
      "model/transformer/layer_2/rel_attn/v/kernel\n",
      "model/transformer/layer_2/rel_attn/r/kernel\n",
      "model/transformer/layer_2/rel_attn/o/kernel\n",
      "model/transformer/layer_2/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_2/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_2/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_2/ff/layer_1/bias\n",
      "model/transformer/layer_2/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_2/ff/layer_2/bias\n",
      "model/transformer/layer_2/ff/LayerNorm/beta\n",
      "model/transformer/layer_2/ff/LayerNorm/gamma\n",
      "model/transformer/layer_3/rel_attn/q/kernel\n",
      "model/transformer/layer_3/rel_attn/k/kernel\n",
      "model/transformer/layer_3/rel_attn/v/kernel\n",
      "model/transformer/layer_3/rel_attn/r/kernel\n",
      "model/transformer/layer_3/rel_attn/o/kernel\n",
      "model/transformer/layer_3/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_3/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_3/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_3/ff/layer_1/bias\n",
      "model/transformer/layer_3/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_3/ff/layer_2/bias\n",
      "model/transformer/layer_3/ff/LayerNorm/beta\n",
      "model/transformer/layer_3/ff/LayerNorm/gamma\n",
      "model/transformer/layer_4/rel_attn/q/kernel\n",
      "model/transformer/layer_4/rel_attn/k/kernel\n",
      "model/transformer/layer_4/rel_attn/v/kernel\n",
      "model/transformer/layer_4/rel_attn/r/kernel\n",
      "model/transformer/layer_4/rel_attn/o/kernel\n",
      "model/transformer/layer_4/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_4/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_4/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_4/ff/layer_1/bias\n",
      "model/transformer/layer_4/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_4/ff/layer_2/bias\n",
      "model/transformer/layer_4/ff/LayerNorm/beta\n",
      "model/transformer/layer_4/ff/LayerNorm/gamma\n",
      "model/transformer/layer_5/rel_attn/q/kernel\n",
      "model/transformer/layer_5/rel_attn/k/kernel\n",
      "model/transformer/layer_5/rel_attn/v/kernel\n",
      "model/transformer/layer_5/rel_attn/r/kernel\n",
      "model/transformer/layer_5/rel_attn/o/kernel\n",
      "model/transformer/layer_5/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_5/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_5/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_5/ff/layer_1/bias\n",
      "model/transformer/layer_5/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_5/ff/layer_2/bias\n",
      "model/transformer/layer_5/ff/LayerNorm/beta\n",
      "model/transformer/layer_5/ff/LayerNorm/gamma\n",
      "model/transformer/layer_6/rel_attn/q/kernel\n",
      "model/transformer/layer_6/rel_attn/k/kernel\n",
      "model/transformer/layer_6/rel_attn/v/kernel\n",
      "model/transformer/layer_6/rel_attn/r/kernel\n",
      "model/transformer/layer_6/rel_attn/o/kernel\n",
      "model/transformer/layer_6/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_6/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_6/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_6/ff/layer_1/bias\n",
      "model/transformer/layer_6/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_6/ff/layer_2/bias\n",
      "model/transformer/layer_6/ff/LayerNorm/beta\n",
      "model/transformer/layer_6/ff/LayerNorm/gamma\n",
      "model/transformer/layer_7/rel_attn/q/kernel\n",
      "model/transformer/layer_7/rel_attn/k/kernel\n",
      "model/transformer/layer_7/rel_attn/v/kernel\n",
      "model/transformer/layer_7/rel_attn/r/kernel\n",
      "model/transformer/layer_7/rel_attn/o/kernel\n",
      "model/transformer/layer_7/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_7/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_7/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_7/ff/layer_1/bias\n",
      "model/transformer/layer_7/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_7/ff/layer_2/bias\n",
      "model/transformer/layer_7/ff/LayerNorm/beta\n",
      "model/transformer/layer_7/ff/LayerNorm/gamma\n",
      "model/transformer/layer_8/rel_attn/q/kernel\n",
      "model/transformer/layer_8/rel_attn/k/kernel\n",
      "model/transformer/layer_8/rel_attn/v/kernel\n",
      "model/transformer/layer_8/rel_attn/r/kernel\n",
      "model/transformer/layer_8/rel_attn/o/kernel\n",
      "model/transformer/layer_8/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_8/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_8/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_8/ff/layer_1/bias\n",
      "model/transformer/layer_8/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_8/ff/layer_2/bias\n",
      "model/transformer/layer_8/ff/LayerNorm/beta\n",
      "model/transformer/layer_8/ff/LayerNorm/gamma\n",
      "model/transformer/layer_9/rel_attn/q/kernel\n",
      "model/transformer/layer_9/rel_attn/k/kernel\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model/transformer/layer_9/rel_attn/v/kernel\n",
      "model/transformer/layer_9/rel_attn/r/kernel\n",
      "model/transformer/layer_9/rel_attn/o/kernel\n",
      "model/transformer/layer_9/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_9/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_9/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_9/ff/layer_1/bias\n",
      "model/transformer/layer_9/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_9/ff/layer_2/bias\n",
      "model/transformer/layer_9/ff/LayerNorm/beta\n",
      "model/transformer/layer_9/ff/LayerNorm/gamma\n",
      "model/transformer/layer_10/rel_attn/q/kernel\n",
      "model/transformer/layer_10/rel_attn/k/kernel\n",
      "model/transformer/layer_10/rel_attn/v/kernel\n",
      "model/transformer/layer_10/rel_attn/r/kernel\n",
      "model/transformer/layer_10/rel_attn/o/kernel\n",
      "model/transformer/layer_10/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_10/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_10/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_10/ff/layer_1/bias\n",
      "model/transformer/layer_10/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_10/ff/layer_2/bias\n",
      "model/transformer/layer_10/ff/LayerNorm/beta\n",
      "model/transformer/layer_10/ff/LayerNorm/gamma\n",
      "model/transformer/layer_11/rel_attn/q/kernel\n",
      "model/transformer/layer_11/rel_attn/k/kernel\n",
      "model/transformer/layer_11/rel_attn/v/kernel\n",
      "model/transformer/layer_11/rel_attn/r/kernel\n",
      "model/transformer/layer_11/rel_attn/o/kernel\n",
      "model/transformer/layer_11/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_11/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_11/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_11/ff/layer_1/bias\n",
      "model/transformer/layer_11/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_11/ff/layer_2/bias\n",
      "model/transformer/layer_11/ff/LayerNorm/beta\n",
      "model/transformer/layer_11/ff/LayerNorm/gamma\n",
      "Tensor(\"model/transformer/dropout_2/Identity:0\", shape=(?, ?, 768), dtype=float32)\n",
      "WARNING:tensorflow:From <ipython-input-14-7df9c2cf3ecb>:14: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-14-7df9c2cf3ecb>:14: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "WARNING:tensorflow:From <ipython-input-12-9bba20c7f0a3>:46: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "the_inp_tokens, the_inp_mask, the_out_chart, the_out_tags, bert_features = make_network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parse_nk import BERT_TOKEN_MAPPING\n",
    "def bertify_batch(sentences):\n",
    "    all_input_ids = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_input_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_word_end_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "\n",
    "    subword_max_len = 0\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        tokens = []\n",
    "        word_end_mask = []\n",
    "\n",
    "        cleaned_words = []\n",
    "        for word in sentence:\n",
    "            word = BERT_TOKEN_MAPPING.get(word, word)\n",
    "            if word == \"n't\" and cleaned_words:\n",
    "                cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "                word = \"'t\"\n",
    "            cleaned_words.append(word)\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = parser.bert_tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_end_mask.append(0)\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(\"<sep>\")\n",
    "        word_end_mask.append(1)\n",
    "        tokens.append(\"<cls>\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = parser.bert_tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(input_ids)\n",
    "\n",
    "        subword_max_len = max(subword_max_len, len(input_ids))\n",
    "\n",
    "        all_input_ids[snum, :len(input_ids)] = input_ids\n",
    "        all_word_end_mask[snum, :len(word_end_mask)] = word_end_mask\n",
    "        all_input_mask[snum, :len(input_mask)] = input_mask\n",
    "\n",
    "    all_input_ids = all_input_ids[:, :subword_max_len]\n",
    "    all_word_end_mask = all_word_end_mask[:, :subword_max_len]\n",
    "    all_input_mask = all_input_mask[:, :subword_max_len]\n",
    "    return all_input_ids, all_word_end_mask, all_input_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 39, 768)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subbatch_trees = test_treebank[:2]\n",
    "subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "inp_val_tokens, inp_val_mask, input_mask = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "sess.run(bert_features, {the_inp_tokens: inp_val_tokens, the_inp_mask: input_mask}).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:03<00:00,  7.30it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "eval_batch_size = 16\n",
    "test_predicted = []\n",
    "for start_index in tqdm(range(0, len(test_treebank), eval_batch_size)):\n",
    "    subbatch_trees = test_treebank[start_index:start_index+eval_batch_size]\n",
    "    subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "    inp_val_tokens, inp_val_mask, _ = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "    out_val_chart, out_val_tags = sess.run((the_out_chart, the_out_tags), \n",
    "                                       {the_inp_tokens: inp_val_tokens, the_inp_mask: inp_val_mask})\n",
    "    trees = []\n",
    "    scores = []\n",
    "    for snum, sentence in enumerate(subbatch_sentences):\n",
    "        chart_size = len(sentence) + 1\n",
    "        tf_chart = out_val_chart[snum,:chart_size,:chart_size,:]\n",
    "        sentence = list(zip([TAG_VOCAB[idx] for idx in out_val_tags[snum,1:chart_size]], [x for x in sentence]))\n",
    "        tree, score = parser.decode_from_chart(sentence, tf_chart)\n",
    "        trees.append(tree)\n",
    "        scores.append(score)\n",
    "    test_predicted.extend([p.convert() for p in trees])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7f357c5ec358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa10eb70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c602dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa1046d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c5a2c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa114240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c5c19e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa114d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0996a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa117898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c09bf60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa11a320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c09e860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa11acc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0a3208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa11c710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0a4cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa11f3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0a8940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa11ff28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0ad550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa122c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0b2160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa125898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0b1cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa128518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0b6940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0ab198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0bb550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0abe48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0bdf60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0adb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c040ba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa109470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c043d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa10b208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c047a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa10b780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c04a4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0b1e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c04bcf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa1024e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c04ec18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa102ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c04fb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f7588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c051a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f7c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c052978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa105278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c055860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa105940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c056748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa108128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c059668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa1087f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c05c588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa108eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c05d898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0b65c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c13c3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0b6f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c13ce80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0bc4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c13b978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0b7390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c1374a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0b7828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c137f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0b7d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c133ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0c01d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0cd588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0c0668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0d8080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0c0b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0d4128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0c0f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0cc160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0c3d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0c1278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0c7a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c05e438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0ca8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0605f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0cc240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0627f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0ccb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c064a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0cf4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c066c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0cfe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c068dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0d3780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c06afd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0d50b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c06e278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f3668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c071208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f3e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c075dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0d8b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c077da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0d9e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c079da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0df4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c07cda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0dfb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c07ddd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e21d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c07fd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e2860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c001d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e2e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c003d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e5550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c005da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e5be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0088d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e8320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c00d438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0e8f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c00db38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa06bb70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c00fb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa06bef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c013080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa06ddd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0135f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fd198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c013b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fd470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c014048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fd780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c014588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fd668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c014b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fdc88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0160f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fdfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c016630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa1005f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c01abe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa100048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c01dcf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa154f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c021780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0d5f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c022240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f32e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c022c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa10e668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c024710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa10e278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c026198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa10e9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c026be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0714e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c028668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0719b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0290f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0882e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c02c6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0887f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c02cef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08b2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c02e780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08b668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c02f080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08ba20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c02f940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08bdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c0321d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08f1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c032a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08f588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c033390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08f940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c033c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa08fcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c037400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0910f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357c03ba90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa091c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354439c198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa093e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443a0748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0befd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443a3e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa09b198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443aa518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa09e358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443b0b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa031b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443b5240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa035cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443b6860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0397f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443b9e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa03b1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443bd588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa03bbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443bebe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa03e6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443c3358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0420b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443c59b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa042b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443ca080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa044588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443cb7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa044fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443cff60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa047a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443d7710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa04ae80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443daef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa052390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544361630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa054898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544366e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa056da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354436c630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa05c278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354436e470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa085b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544371278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa085f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443730b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0a1ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544373eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0a1f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544375cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0a80f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544378be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0a8cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544379ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0aa978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354437b9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0aacf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354437f320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa02cc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544381c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa02c7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544386550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa02f240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544387dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa060b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354438c780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0656a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443910f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa068160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544392a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa068c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544396400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7b9828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544396a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07dbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443990f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07d908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544399748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07deb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544399d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07ea90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f357ce80198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07e1d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354439a978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07eda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354439afd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07eeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354431c668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07edd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544321828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07ec88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443253c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa081dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544326ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7bcac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354432f208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0834a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544333160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7c2ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443336a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7c5748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544333be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7c5940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544335160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7c5b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544335668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cd4a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544335b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cd710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443370f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cd8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443375f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cdac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544337b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cdcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354433d5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cdeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354433f550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d0320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544342518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d0a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544343400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d1160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544345320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d1860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35443482b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d1f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354434a208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d56a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354434c128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d5da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354434e080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d84e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354434f8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7d8be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544355240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7da5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544356be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7f02b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354435a710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7f0cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442df0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7f4710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442e0ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7f70f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442e5588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7f7ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442e6ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c77b518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442ec438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c77beb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442efdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c77def0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442f3780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c781b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442f7198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7846d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442f9b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7862e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442fe550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c786eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442fff28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c78ab00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544303908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c78e710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544308320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c790320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354430c860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c790ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544310d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c79df28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544316320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7a0198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354429b860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7a4358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354429ed68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7a74e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442a6390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7a8668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442a9940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7ab7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442adf28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7ae978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442b2518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7b1b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442b4b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7b54e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442b9198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7b5e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442ba710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7b87f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442bdd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c73b198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442c0390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c73bb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442c3940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c73e470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442c5f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c73eeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442c9630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c73f860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442cb438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c751cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442ce208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c754320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442cefd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c754908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442d0da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c754f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442d2be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c757668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442d3a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c757c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442d5860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c75a278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354425bf60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c75a860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442656d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c761198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544269da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c764a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544273518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c76b390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544278cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c76dc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442813c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c775588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544287b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c776dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544290278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6ff748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544293e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c705080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544299a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c796518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544220668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c79d9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544227208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7418d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354422bdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c749cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442319b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7087f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544238588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c70e8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354423e1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c70fc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354423ecc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c712f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544241828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c715438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544242390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c715898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544242eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c715cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544245a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c718160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442465c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c718588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35442480f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7189e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544248c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c718eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354424d8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c71c320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544250518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7305c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544256198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c734358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544257e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7360f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441dbac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c736e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441e16a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6baba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441e43c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6bc908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441e6fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6be6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441eac88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6c3470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441efba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6c8208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441f3a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6c8f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441fd390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6c9d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544201b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6d0710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354420b3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6d50b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544211b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6da9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354421a390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6df2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354419fb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6e3ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441aa320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c70a6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441b0b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c727240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441b5358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c72b8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441b8c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6e9e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441be630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6f5208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441c3f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6f7588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441c9828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c679908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441cf160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c67dc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441d3ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c680f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441d6e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c687358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441d9208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c687be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354415d550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6884a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354415e940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c688d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544160b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c68a668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544162cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c68af28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544165e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c68d7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544167fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c69cc88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354416b1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c69f4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354416c390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c69fcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441744a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6a25f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354417ba20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6a82b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544185048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6a9ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544189588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6b0588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544190ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6b3f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354419a198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c63a9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354411e828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c640358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544123e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c644cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354412b2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c646e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354412f2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c64e6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544131b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c64f518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544135438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6531d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544137cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c653e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354413c5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7e8080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354413de10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7ead68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441436a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7ee208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544144f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7cd048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544147828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c744518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354414acf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c70a3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354414f2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6913c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441517b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c691f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544153c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c693898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441571d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6980b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441586a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c698b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440dbb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c656f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440e00b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c65b9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440e4080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c65d400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440e8048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c660278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440e9fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6620f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440edfd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c66a588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440f1f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c66c240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440f5f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c66ceb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440fbe80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c670b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440fee80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6729b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544102cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c676828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441072e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5fa588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544109588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5fafd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354410a358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5fc860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544110c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5ff5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544113978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5fff60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35441176a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c602ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354409c358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6057f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354409df98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c608438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440a2cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c60c0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440a69e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c60cc88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440ab208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7ded30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440aea20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c662f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440b22b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6593c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440b3a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c60fe80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440b9278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c615978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440baa58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c619470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440bf320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c619f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440c1b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c61ca90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440c6080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c61f5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440c75c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c61ff98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440cab70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6229b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440ce198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c624390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440cf6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c624d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440d2c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c6277f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440d6278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c62a240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440d8898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c62abe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354405cb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c637080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544060dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c638048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440680b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c638fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354406c2e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5baf98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544070518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c613f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544072668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5c4048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544078470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5c47b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354407a978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5c6cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354407de80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5c9630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544080358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5c9eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440837f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5cd7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544085c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5cf128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440891d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5cfa58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354408b748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5d2358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354408ccc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5d2c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544091e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5d5588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544094fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c635940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354401c128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5d8ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544020240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5db940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544024358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5de908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544028470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5e17f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354402c588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5e4748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544030748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5e7710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544032b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5ea5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544034eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5eaef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440392b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5ed7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354403a630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5ef128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354403c9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5efa20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354403ed68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5f3358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544043128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5f3c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440444a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5f7550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544047eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5f7e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f354404bfd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c58ce48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544051240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c58ff60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35440564e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c592f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3544059780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c594fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147bfa90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c599080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147c4c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c599fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147c7eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5a10f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147ce208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5a3128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147d28d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5a60b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147d9470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5a9128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147dcac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5ad358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147e31d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5b04e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147e7828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5b16d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147eaf28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5b48d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147f15c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5b8ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147f5c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c53ac18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147fa128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c62ecf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147fc588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c66a3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351477da20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c580128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351477fe48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c580c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147842b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c57f358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514785710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c57fe10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514788ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c586630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351478af28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c58ada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147903c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c58ab38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514791eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c7de198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147987b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c543668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351479e080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5446d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147a25f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c54a748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147a47b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c54f6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147a8358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c54fe48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147a9860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5519e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147ad080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f90b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147ad860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f9438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147af080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0f9ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147af860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fd2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147b1080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa0fd048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147b17f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa071eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147b1fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07de10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147b37f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07d860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147bb2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f34aa07d390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147bce48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c55b198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514741240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c55bd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514743e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c55d7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514748a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c55f630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351474c6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c562438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514750320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5662e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514752f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c56a0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514757b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c4fbcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351475b7b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c4fea90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351475e438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c501908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147645c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c504748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514767630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5076d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351476c780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c50a588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147707f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c50d438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514774828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c510358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514778940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c57d3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35146fdac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5112e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35146ffd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c514278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514704198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c514be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147065f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c5175c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147089b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c517f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351470acf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c51a8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351470e128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c51d2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147114a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c56dc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514713828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c574518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514719470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c574a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351471cb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c4fb358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514720320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c4fb048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514721ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c522400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514727240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c522e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147289e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c524a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351472c208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c527518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351472f9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c527f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514732208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c52aa58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514732da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c52c518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514735940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c52ca58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35147374e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c52cfd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514739080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c52f550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f3514739c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c52f978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f351473b7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c635588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35146bd358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c58c2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7f35146bdef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7f344c58c208>\n"
     ]
    }
   ],
   "source": [
    "test_fscore = evaluate.evalb('EVALB/', test_treebank[:len(test_predicted)], test_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(Recall=81.52, Precision=85.18, FScore=83.31, CompleteMatch=11.71, TaggingAccuracy=91.71)'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_names = [the_inp_tokens.name.split(':')[0], the_inp_mask.name.split(':')[0]]\n",
    "output_node_names = [the_out_chart.name.split(':')[0], the_out_tags.name.split(':')[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-22-7c3651befba4>:1: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 161 variables.\n",
      "INFO:tensorflow:Converted 161 variables to const ops.\n"
     ]
    }
   ],
   "source": [
    "graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, output_node_names)\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names, [\n",
    "'strip_unused_nodes()',\n",
    "'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "'fold_constants()',\n",
    "'fold_old_batch_norms',\n",
    "'fold_batch_norms',\n",
    "'round_weights(num_steps=128)',\n",
    "])\n",
    "with open('export/xlnet-base.pb', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names,\n",
    "                          transforms = \n",
    "                           ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_constants(ignore_errors=true)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order'])\n",
    "with open('export/xlnet-base.pb.quantized', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab-xlnet-base.json', 'w') as fopen:\n",
    "    json.dump({'label': LABEL_VOCAB, 'tag': TAG_VOCAB}, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
